# coding: utf-8
import urllib
import urllib2
import cookielib
from HTMLParser import HTMLParser
import re
import xlrd
import xlwt
import string
from urllib2 import Request, urlopen, URLError, HTTPError
import requests

#=====================================================

def getStrvalue(tar,strname,spstr="&nbsp;"):
    if tar.find(strname)>=0:
        ress=tar.split(strname)
        print "可以切割"+strname+"   切割后长度："+str(len(ress))
        if len(ress)>1:
            ress = ress[1].split(spstr)
            ress = ress[1].split("<")
            return ress[0]
        else:
            print "切割出错，字符串没有："+strname
    return ""


#======================================================
class myHtmlParser(HTMLParser):
    irstart=0
    irend=0
    result=[]
    data=[]
    isfinids=0
    iscleasspace=0
    def __init__(self):
        HTMLParser.__init__(self)
        self.flag=None
    def handle_starttag(self,tag,attrs):
        if self.irstart==1:
            if tag=='a':
                self.flag='a'
                for href,link in attrs:
                    if href=='href':
                        if self.isfinids==1:
                            if link!='/newsAction.do?method=viewNews&classId=020019980000000472&newsId=020010040000016998' and link!='/newsAction.do?method=viewNews&classId=020010350000000877&newsId=020010040000108745':
                                if link.find('/newsAction.do?method=viewNews&classId')>=0:
                                    self.result.append(link)
                        else:
                            self.result.append(link)





    def handle_data(self,data):
        if self.irend==1:
            sdata = data.strip()
            if sdata!='':
                if self.iscleasspace==1:
                    sdata=sdata.replace("\r\n","")
                    sdata=sdata.replace(" ","")
                    sdata=sdata.replace(' ','')
                self.data.append(sdata)
                #print 'data:'+sdata

#======================================================
#http://www.gdlr.gov.cn/newsAction.do?method=viewNews&classId=020010350000000680&newsId=020010040000205418
fdcinfolist=[]

findpage = 1
while findpage<=6:
    print str(findpage)+"获取所有id"
    cookie = cookielib.CookieJar()
    opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
    url='http://www.gdlr.gov.cn/newsAction.do?method=searchNews'
    tilename = u'高明区'
    tilename = tilename.encode("gb2312")
    values={
        'classId':'null',
        'textGeneralType':'020010350000000678',
        'newsCatalogID':'020010350000000678',
        'isquery':'',
        'type':'',
        'type_name':'',
        'oldTitle':tilename,'oldTitle':'','titleName':tilename,'linkUrl':'',
        '020010350000000678':'','strTime':'','endTime':'','orderSign':'0','curPageNo':'1',
        'totalCnts':'54','totalPages':'6','cntPerPage':'10','SplitFlag':'1','orderBy':'null','descOrAsc':'desc','gotoPage':findpage}
    data = urllib.urlencode(values)
    req = urllib2.Request(url,data)
    req.add_header('Host','www.gdlr.gov.cn')
    req.add_header('Referer','http://www.gdlr.gov.cn/newsAction.do?method=searchNews&textGeneralType=020010350000000678')
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')
    response = opener.open(req)
    the_page=response.read()
    pstrl = unicode(the_page,"gb2312",'ignore').encode("utf8")
    response.close()
    m= myHtmlParser()
    m.irstart=1
    m.isfinids=1
    m.feed(pstrl)
    m.close()
    result = m.result
    for itemstr in result:
        fdcinfolist.append(itemstr)
    findpage+=1

print "删除重复前:"+str(len(fdcinfolist))
fdcinfolist =  list(set(fdcinfolist))
print "删除重复后:"+str(len(fdcinfolist))

allDic=[]
#创建表格
f= xlwt.Workbook()#创建工作簿
sheetname = u"结果出让公告"
sheet = f.add_sheet(sheetname,cell_overwrite_ok=True)
#行头
print "创建表格"

readPos = -1
pppppp=1
for fdcid in fdcinfolist:
    readPos+=1
    print  "=获取id:"+fdcid
    url = 'http://www.gdlr.gov.cn'+fdcid
    print '浏览网页:'+url
    bskk =0
    botiaozhuan = 0
    try:
        headers = requests.head(url).headers

        for kkk in headers:
            if kkk=='location':
                url= headers['location']
                botiaozhuan=1
    except :
        botiaozhuan=0


    request = urllib2.Request(url)
    print '跳转网页:'+url
    if botiaozhuan==1:
        request.add_header('Host','www.landgd.com')
    else:
        request.add_header('Host',' www.gdlr.gov.cn')
    request.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')

    try:
        response = urllib2.urlopen(request)
    except :
        continue

    print str(readPos)+"读取网址"+url

    strl = response.read()
    response.close()
    pstrl = unicode(strl,"gb2312",'ignore').encode("utf8")
    strl = pstrl
    print strl
    strl=strl.replace("\r\n","")
    strl=strl.replace(" ","")
    p=re.compile('\s+')
    strl=re.sub(p,'',strl)
    s="====================="
    lencount =2
    canread = 0
    if strl.find("一、")>=0 and strl.find("二、"):
        canread =1

    if strl.find("工业用地")>=0 or strl.find("商服用地")>=0 or strl.find("公共设施用地")>=0:
        canread=0
    if canread>0:
        results = strl.split("一、")
        print "len1:"+str(len(results))
        results = results[1].split("二、")
        print "len2:"+str(len(results))

        print "================================="
        print strl
        #获取发布时间
        sltimes = strl.split("发布时间：")
        ptime=""
        if len(sltimes)>1:
            nstr = sltimes[1]
            sltimes = nstr.split("<")
            ptime=sltimes[0]
        print "发布时间："+ptime
        print "================================="
        print results[0]
        strl=results[0]
        objDic=[]
        objDic.append(ptime)
        objDic.append(getStrvalue(strl,"宗地编号"))
        objDic.append(getStrvalue(strl,"宗地面积"))
        objDic.append(getStrvalue(strl,"宗地坐落"))
        objDic.append(getStrvalue(strl,"出让年限"))
        objDic.append(getStrvalue(strl,"容积率："))
        objDic.append(getStrvalue(strl,"建筑密度(%)"))
        objDic.append(getStrvalue(strl,"绿化率(%)"))
        objDic.append(getStrvalue(strl,"建筑限高(米)"))
        objDic.append(getStrvalue(strl,"土地用途"))
        objDic.append(getStrvalue(strl,"投资强度"))
        objDic.append(getStrvalue(strl,"保证金："))
        objDic.append(getStrvalue(strl,"估价报告备案号：","<td>"))
        objDic.append(getStrvalue(strl,"起始价："))
        objDic.append(getStrvalue(strl,"加价幅度"))
        objDic.append(getStrvalue(strl,"挂牌开始时间"))
        objDic.append(getStrvalue(strl,"挂牌截止时间"))
        objDic.append(getStrvalue(strl,"备注："))
        allDic.append(objDic)
        print "==============================="
        print "  "
        print "公布时间:"+ptime
        print "宗地编号"+":"+objDic[1]
        print "宗地面积"+":"+objDic[2]
        print "宗地坐落"+":"+objDic[3]
        print "出让年限"+":"+objDic[4]
        print "容积率"+":"+objDic[5]
        print "建筑密度"+":"+objDic[6]
        print "绿化率"+":"+objDic[7]
        print "建筑限高"+":"+objDic[8]
        print "土地用途"+":"+objDic[9]
        print "投资强度"+":"+objDic[10]
        print "保证金"+":"+objDic[11]
        print "估价报告备案号"+":"+objDic[12]
        print "起始价"+":"+objDic[13]
        print "加价幅度"+":"+objDic[14]
        print "挂牌开始时间"+":"+objDic[15]
        print "挂牌截止时间"+":"+objDic[16]
        print "备注"+":"+objDic[17]

f= xlwt.Workbook()#创建工作簿
sheetname = "出让公告"
sheet = f.add_sheet(unicode(sheetname,'utf-8'),cell_overwrite_ok=True)
#创建表头
sheet.write(0,0,u'公布时间')
sheet.write(0,1,u'宗地编号')
sheet.write(0,2,u"宗地面积")
sheet.write(0,3,u"宗地坐落")
sheet.write(0,4,u"出让年限")
sheet.write(0,5,u"容积率%")
sheet.write(0,6,u"建筑密度%")
sheet.write(0,7,u"绿化率%")
sheet.write(0,8,u"建筑限高")
sheet.write(0,9,u"土地用途")
sheet.write(0,10,u"投资强度")
sheet.write(0,11,u"保证金")
sheet.write(0,12,u"估价报告备案号")
sheet.write(0,13,u"起始价")
sheet.write(0,14,u"加价幅度")
sheet.write(0,15,u"挂牌开始时间")
sheet.write(0,16,u"挂牌截止时间")
sheet.write(0,17,u"备注")


nrows = len(allDic)
i=0
while i<nrows:
    wdic = allDic[i]
    k=i+1
    print "新的数据，开始写入档案"
    wcount = len(wdic)
    wi = 0
    while wi<wcount:
        sheet.write(k,wi,unicode(wdic[wi],'utf-8'))
        wi+=1
    i+=1

f.save( u'高明土地信息（出让公告）.xls')